#coding=utf-8

from scrapy.spiders import Spider
from scrapy.selector import Selector
from scrapy.linkextractors import LinkExtractor
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.spiders import Rule,CrawlSpider
from zhihuSpider.items import ZhihuspiderItem
from scrapy.http import Request, FormRequest
import Queue
from scrapy.http.cookies import CookieJar
import os
import time

class ZhihuMessageSpider(CrawlSpider):
    name = 'zhihu_message_spider'

    allowde_domains = ['zhihu.com']
    start_urls = [
        'https://www.zhihu.com'
    ]
    url_queue = Queue.Queue()
    

    rules = (
        # Rule(SgmlLinkExtractor(allow=('https://www.zhihu.com/people',), )),
        Rule(SgmlLinkExtractor(allow=(r'https://www.zhihu.com/people/')), callback='parse_page', follow=True),
    )

    headers = {
        "Accept": "*/*",
        "Accept-Encoding": "gzip,deflate",
        "Accept-Language": "en-US,en;q=0.8,zh-TW;q=0.6,zh;q=0.4",
        "Connection": "keep-alive",
        "Content-Type": " application/x-www-form-urlencoded; charset=UTF-8",
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36",
        "Referer": "http://www.zhihu.com/"
    }

    # 重写了爬虫类的方法, 实现了自定义请求, 运行成功后会调用callback回调函数
    def start_requests(self):
        return [Request("https://www.zhihu.com/#signin",headers=self.headers, meta={'cookiejar': 1}, callback=self.post_login)]

    # FormRequeset出问题了
    def post_login(self, response):
        print 'Preparing login'
        # 下面这句话用于抓取请求网页后返回网页中的_xsrf字段的文字, 用于成功提交表单
        xsrf = Selector(response).xpath('//input[@name="_xsrf"]/@value').extract()[0]
        print xsrf
        captcha_url = "http://www.zhihu.com/captcha.gif?r=" + str(time.time() * 1000)
        # FormRequeset.from_response是Scrapy提供的一个函数, 用于post表单
        # 登陆成功后, 会调用after_login回调函数
        return [FormRequest.from_response(response,  # "http://www.zhihu.com/login",
                                          meta={'cookiejar': response.meta['cookiejar']},
                                          headers=self.headers,  # 注意此处的headers
                                          formdata={
                                              '_xsrf': xsrf,
                                              'email': 'lck5602@126.com',
                                              'password': '19891109#',
                                              'remember_me':'true'
                                          },
                                          callback=self.after_login,
                                          dont_filter=True
                                          )]

    def after_login(self, response):
        for url in self.start_urls:
            yield Request(url=url,
                          headers=self.headers,
                          meta={'cookiejar': response.meta['cookiejar']},
                          callback=self.parse_page,
                          dont_filter=True
                          )
            # yield self.make_requests_from_url(url)

    def parse_page(self,response):
        sel = Selector(response)

        user_name = sel.xpath("//span[@class='name']/text()").extract()
        user_location = sel.xpath("//span[@class='location item']/@title").extract()
        user_gender = sel.xpath("//span[@class='item gender']/i/@class").extract()
        if user_gender and user_gender[0] and 'female' in user_gender[0].split('-'):
            user_gender ='female'
        else:
            user_gender = 'male'
        user_employment = sel.xpath("//span[@class='employment item']/@title").extract()
        user_employment_extra = sel.xpath("//spam[@class='position item']/@title").extract()
        user_education_school = sel.xpath("//span[@class='education item']/@title").extract()
        user_education_subject = sel.xpath("//span[@class='education-extra item']/@title").extract()
        try:
            user_followees = sel.xpath("//div[@class='zu-main-sidebar']//strong/text()")[0].extract()
            user_followers = sel.xpath("//div[@class='zu-main-sidebar']//strong/text()")[1].extract()
        except:
            user_followees = ''
            user_followers = ''
        user_be_aggred = sel.xpath("//span[@class='zm-profile-header-user-agree']//strong/text()").extract()[0]
        user_be_thanked = sel.xpath("//span[@class='zm-profile-header-user-thanks']//strong/text()").extract()[0]
        user_info = ','.join(sel.xpath("//span[@class='bio']/@title").extract())
        user_intro = ','.join(sel.xpath("//span[@class='content']/text()").extract())
        urls_list = sel.xpath("//div[@class='author-info']//a/@href").extract()

        items = ZhihuspiderItem()

        items['user_name'] = user_name[0] if user_name else ''
        items['user_gender'] = user_gender
        items['user_location'] = user_location[0] if user_location else ''
        items['user_followees'] = user_followees
        items['user_followers'] = user_followers
        items['user_be_aggred'] = user_be_aggred
        items['user_be_thanked'] = user_be_thanked
        items['user_education_school'] = user_education_school[0] if user_education_school else ''
        items['user_education_subject'] = user_education_subject[0] if user_education_subject else ''
        items['user_employment'] = user_employment[0] if user_employment else ''
        items['user_employment_extra'] = user_employment_extra if user_employment_extra else ''
        items['user_info'] = user_info
        items['user_intro'] = user_intro
        for url in urls_list:
            self.url_queue.put(url)
        yield items
        print user_name,user_gender

        #
        # if not self.url_queue.empty():
        #     url =  self.url_queue.get()
        #     url = 'https://www.zhihu.com' + url
        #     yield Request(url, headers=self.headers, callback=self.parse_page)